import tweepy
from tweepy.parsers import RawJsonParser
from tweepy.streaming import StreamListener, Stream
import logging
import json
import sys
import os
from datetime import datetime
import time
from multiprocessing import Process, Value, Lock

class CrawlerReporter:
    def __init__(self):
        self._t = None
        self._lock = None
        pass
    
    def SetReport(self, ft, lock):
        self._t = ft
        self._lock = lock
    
    def Report(self, value = -1):
        with self._lock:
            if value == -1:
                self._t.value = time.time()
            else:
                self._t.value = value

class TwitterListener ( StreamListener):
    def __init__(self, working_dir, crawl_reporter):
        StreamListener.__init__(self)
        self._dir = working_dir
        self._output_filename = ''
        self._output_filehandler = None
        self._r = crawl_reporter
    
    def _get_output_filename(self):
        current_time = datetime.now()
        filename = str(current_time.year) + '_' + str(current_time.month) + '_' + str(current_time.day) + '_' + str(current_time.hour)
        return filename
    
    def _get_output_file(self):
        should_filename = self._get_output_filename()
        if should_filename != self._output_filename:
            if self._output_filehandler != None:
                self._output_filehandler.close()
            self._output_filehandler = open(os.path.join(self._dir,should_filename),'a')
            logging.info('Start to save file in ' + should_filename)
        self._output_filename = should_filename
        return self._output_filehandler
        

    def on_data(self, data):
        if 'id' in data:
            if not 'delete' in data:
                cfile = self._get_output_file()
                cfile.write(data.encode('utf-8')+'\n')
                cfile.flush()
                self._r.Report()
        return
    
    def on_error(self, status_code):
        print >> sys.stderr, 'Encountered error with status code:', status_code
        return True # Don't kill the stream

    def on_timeout(self):
        print >> sys.stderr, 'Timeout...'
        return True # Don't kill the stream

class TwitterCrawler:
    global_auth = None
    auth_key = ''
    non_json_api = None
    json_api = None
    MAX_TRIAL = 3
    
    def ListenStreamSample(self, working_dir, r):
        l = TwitterListener(working_dir, r)
        try:
            streamer = tweepy.Stream(auth=self.global_auth, listener=l)
            streamer.sample()
        except:
            logging.warn('error in monitor.')
            r.Report(0.0)
        return
    
    def __init__(self, key = 'day'):
        self.auth_key = key
        self.PlainTextFile = ''
        self.Reporter = None
        
    def Auth(self, key = 'day'):
        #================= all keys=========================================
        keys = {}
        keys['day'] = {}
        keys['wume7'] = {}
        keys['wume7']['consumer_key']="GgHhXB28a7luGPdSI55w4A"
        keys['wume7']['consumer_secret']="mpgBLutmZh2CHvYcofUr4vzAeyDjExS3GgBEplsrA"
        keys['wume7']['access_token']="24964836-Ps4rH7nocw6nQ7XKkB3EWABWW5Sy8lPDB52wb6NGp"
        keys['wume7']['access_token_secret']="nWeTMcO1SH0jaEDR4PO7b0jfkgaVzssbbEhR8NmEjk"
        keys['day']['consumer_key'] = "wfkIDPLTX99mFGbhsaIQw"
        keys['day']['consumer_secret'] = "DDrUvdKPlVGcDF1MC7hcJp5cWu4A8aL0HESa8qno3JU"
        keys['day']['access_token']="24964836-sRDtPmFn5cB5WAAjWkq3uU9xs5JBqzoosEeGhjR14"
        keys['day']['access_token_secret']="x5GjaShhuFS4B6audBYiAVK2dCm07syTC1LPfSn6xqM"
        #===================================================================
        logging.info('starting to authorization from Twitter')
        self.global_auth = tweepy.OAuthHandler(keys[key]['consumer_key'], keys[key]['consumer_secret'])
        self.global_auth.set_access_token(keys[key]['access_token'], keys[key]['access_token_secret'])
        logging.info('finish to authorization from Twitter')
    
    def GetAPI(self, json_open = 0):
        api = None
        if json_open == 0:
            if self.non_json_api == None:
                self.non_json_api = tweepy.API(auth_handler= self.global_auth)
            api = self.non_json_api
        else:
            if self.json_api == None:
                self.json_api = tweepy.API(auth_handler= self.global_auth, parser=RawJsonParser())
            api = self.json_api
        return api
    
    
    
    def GetRateLimits(self):
        api = self.GetAPI()    
        r = None
        result = {}
        try:
            r = api.rate_limit_status()
            n = r['remaining_hits']
            result['flag'] = 1
            result['n'] = n
            self.Reporter.Report()
        except:
            result['flag'] = -1
            result['n'] = 0
        logging.info('Limits:'+ str(result['n'])+'\tFLAG:'+str(result['flag']))    
        return result
    
    def KeepLimits(self):
        while True:
            try:
                logging.info('Trying to get limits...')
                result = self.GetRateLimits()
                if result['n'] > 0:
                    logging.info('Success to get limits.')
                    break
                self.Reporter.Report()
            except:
                logging.warn('Fail to get limits...')
                continue
            logging.warn('Wait 5 seconds...')
            time.sleep(5)
        return True
    
    def GetMe(self):
        api = self.GetAPI()
        logging.info('Trying to get me...')
        if self.KeepLimits():
            current_user = api.get_user("hongliangjie")
        logging.info('Success to get me...')
        return current_user
    
    def Friendship(self, id1, id2):
        api = self.GetAPI()
        result = -1        
        if self.KeepLimits():            
            logging.info('Trying to detect the existence of friendship between id1:' + str(id1) + ' and id2:' + str(id2))
            try:
                result = api.exists_friendship(int(id1),int(id2))
                if result == True:
                    result = 1
                else:
                    result = 0
                logging.info('Success to detect the existence of friendship. RESULT:' + str(result))
            except:
                result = -1
                logging.warn('Fail to detect the existence of friendship.')
        return result
    
    def GetFriends(self, sn, user_id):
        api = self.GetAPI()
        result = None     
        if self.KeepLimits():
            if user_id != '':
                logging.info('Trying to obtain friends of id' + str(user_id))
            else:
                logging.info('Trying to obtain friends of sn' + str(sn))
            try:
                result = api.friends_ids(screen_name = sn, user_id = str(user_id))
                if user_id != '':
                    logging.info('Success to obtain friends of id' + str(user_id))
                else:
                    logging.info('Success to obtain friends of sn' + str(sn))
            except:
                result = None
                if user_id != '':
                    logging.warn('Fail to obtain friends of id' + str(user_id))
                else:
                    logging.warn('Fail to obtain friends of sn' + str(sn))
        if self.PlainTextFile != '' and result != None:
            output_f = open(self.PlainTextFile,'a')
            if user_id != '':
                output_f.write(str(user_id)+'\t')
            else:
                output_f.write(str(sn)+ '\t')
            for uid in result:
                output_f.write(str(uid)+' ')
            output_f.write('\n')
            output_f.close()
            self.Reporter.Report()
        return result

    def GetStatus(self, status_id):
        api = self.GetAPI(1)
        trial = 0
        all_status = None
        while all_status == None and trial < self.MAX_TRIAL:
            try:
                if self.KeepLimits():
                    logging.info('Starting to get status:' + str(status_id))
                    all_status = api.get_status(id = status_id, include_entities = 'true')
                    logging.info('Finish to get status:' + str(status_id))
            except:
                time.sleep(0.5)
            trial = trial + 1
        logging.info('Get Out status API.')
        return all_status
    
    def GetBulkScreenNames(self, names):
        api = self.GetAPI(1)
        trial = 0
        all_status = None
        while all_status == None and trial < self.MAX_TRIAL:
            try:
                if self.KeepLimits():
                    logging.info('Starting to bulk request user profiles by ScreenNames.')
                    all_status = api.lookup_users(screen_names = names)
                    logging.info('Finish to bulk request user profiles by ScreenNames.')
            except:
                time.sleep(0.5)
            trial = trial + 1
        logging.info('Get Out bulk request user profiles API.')
        return all_status
    
    def GetBulkIDs(self, ids):
        api = self.GetAPI(1)
        trial = 0
        all_status = None
        while all_status == None and trial < self.MAX_TRIAL:
            try:
                if self.KeepLimits():
                    logging.info('Starting to bulk request user profiles by IDs.')
                    all_status = api.lookup_users(user_ids = ids)                    
                    logging.info('Finish to bulk request user profiles by IDs.')
            except:
                time.sleep(0.5)
            trial = trial + 1
        logging.info('Get Out bulk request user profiles API.')
        return all_status
    
    def GetUser(self, user_id = '', sn = ''):
        api = self.GetAPI(1)
        all_status = None
        if self.KeepLimits():
            logging.info('Starting to single request user profile.')
            try:
                if user_id != '':
                    all_status = api.get_user(user_id = user_id)
                else:
                    all_status = api.get_user(screen_name = sn)
            except:
                logging.info('Fail to single request user profile.')            
        logging.info('Finish to single request user profile.')
        logging.info('Get Out bulk request user profiles API.')
        return all_status
        
    
    def GetUserTimeline(self, sn = '', user_id = '', max_id = -1):
        api = self.GetAPI(1)
        trial = 0
        all_status = None
        while all_status == None and trial < self.MAX_TRIAL:
            try:
                if max_id == -1:
                    if self.KeepLimits():
                        logging.info('Starting user timeline API.')
                        if user_id != '':
                            all_status = api.user_timeline(id= user_id, count = 200, include_rts = 'true', include_entities = 'true')
                        else:
                            all_status = api.user_timeline(screen_name = sn, count = 200, include_rts = 'true', include_entities = 'true')
                        logging.info('Finish user timeline API.')
                else:
                    if self.KeepLimits():
                        logging.info('Starting user timeline API.')
                        if user_id !='':
                            all_status = api.user_timeline(id= user_id, count = 200, max_id = max_id, include_rts = 'true', include_entities = 'true')
                        else:
                            all_status = api.user_timeline(screen_name = sn, count = 200, max_id = max_id, include_rts = 'true', include_entities = 'true')
                        logging.info('Finish user timeline API.')
                self.Reporter.Report()
            except:
                time.sleep(0.5)
            trial = trial + 1
        logging.info('Get Out user timeline API.')
        return all_status
    
    def GetUserAllTimeline(self, sn, user_id, total_page, func):
        total_tweets = 0
        total_buffer = []
        if self.PlainTextFile != '':
            output_file = open(self.PlainTextFile, 'a')
        if user_id != '':
            logging.info('Trying to get timeline list of user ' + str(user_id) + ' page:0.')
        else:
            logging.info('Trying to get timeline list of user ' + str(sn) + ' page:0.')
        all_status = self.GetUserTimeline(sn, user_id)
        if all_status == None:
            if user_id != '':
                logging.warn('[Fail] Fail to get timeline list of user ' + str(user_id) + ' page:0.')
            else:
                logging.warn('[Fail] Fail to get timeline list of user ' + str(sn) + ' page:0.')
        else:
            if user_id != '':
                logging.info('Success to get timeline list of user ' + str(user_id) + ' page:0.')
            else:
                logging.info('Success to get timeline list of user ' + str(sn) + ' page:0.')
            ## try to parse raw json data
            try:
                s = json.loads(all_status)
            except:
                logging.info('[Fail] Fail to parse json of user ' + str(sn) + ' page:0.')
                return
            total_tweets = total_tweets + len(s)
            ## get the lowest ID
            local_buffer = []
            for i in range(len(s)):
                local_buffer.append(int(s[i]['id']))
            local_buffer = sorted(local_buffer)
            local_buffer.reverse() ## now the ids are sorted from large to small
            if func == None:
                if self.PlainTextFile != '':
                    for i in s:
                        output_file.write(json.dumps(i, sort_keys = False) + '\n')
                else:
                    total_buffer.extend(s)
            else:
                func(s)
                self.Reporter.Report()
            ## determine whether we need to call multiple times
            page = 1
            while len(s) == 200:
                if total_page != -1:
                    if page >= total_page:
                        break
                lowest_id = local_buffer[-1] - 1
                if user_id !='':
                    logging.info('Trying to get timeline list of user ' + str(user_id) + ' page:'+str(page))
                else:
                    logging.info('Trying to get timeline list of user ' + str(sn) + ' page:'+str(page))
                all_status = self.GetUserTimeline(sn = sn, user_id = user_id, max_id = lowest_id)
                if all_status == None:
                    if user_id != '':
                        logging.warn('[Fail] Fail to get timeline list of user ' + str(user_id) + ' page:' + str(page))
                    else:
                        logging.warn('[Fail] Fail to get timeline list of user ' + str(sn) + ' page:' + str(page))
                    break
                else:                    
                    try:
                        s = json.loads(all_status)
                    except:
                        logging.info('[Fail] Fail to parse json of user ' + str(sn) + ' page:' + str(page))
                        return
                    if user_id != '':
                        logging.info('Success to get timeline list of user ' + str(user_id) + ' page:' + str(page))
                    else:
                        logging.info('Success to get timeline list of user ' + str(sn) + ' page:' + str(page))
                    ## get the lowest ID
                    local_buffer = []
                    for i in range(len(s)):
                        local_buffer.append(int(s[i]['id']))
                    local_buffer = sorted(local_buffer)
                    local_buffer.reverse() ## now the ids are sorted from large to small
                    if func == None:
                        if self.PlainTextFile != '':
                            for i in s:
                                output_file.write(json.dumps(i, sort_keys = False) + '\n')
                        else:
                            total_buffer.extend(s)
                    else:
                        func(s)
                        self.Reporter.Report()
                    page = page + 1
                    total_tweets = total_tweets + len(s)
            if self.PlainTextFile != '':
                output_file.close()
        return total_buffer
